{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ef92c15d",
   "metadata": {},
   "source": [
    "## Metrics\n",
    "\n",
    "汇总常见2分类的指标，例如: AUC，ROC曲线，ACC, 敏感性， 特异性，精确度，召回率，PPV, NPV, F1\n",
    "\n",
    "具体的介绍，可以参考一下：https://blog.csdn.net/sunflower_sara/article/details/81214897"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18d9b291",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from datetime import datetime\n",
    "from onekey_algo import get_param_in_cwd\n",
    "\n",
    "os.makedirs('img', exist_ok=True)\n",
    "os.makedirs('results', exist_ok=True)\n",
    "group_info = pd.read_csv(get_param_in_cwd('label_file'))[['ID', 'group']]\n",
    "group_info['ID'] = group_info['ID'].map(lambda x: str(x))\n",
    "display(group_info['group'].value_counts())\n",
    "group_info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a45a469",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy  as np\n",
    "import re\n",
    "from onekey_algo.custom.components import metrics\n",
    "from onekey_algo.custom.components.comp1 import draw_roc, normalize_df\n",
    "from onekey_algo.custom.components.ugly import drop_error\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "def get_log(log_path, map2gz:bool = False):\n",
    "    log_ = pd.read_csv(log_path, names=['fname', 'pred_score', 'pred_label', 'gt'], sep='\\t')\n",
    "    if map2gz:\n",
    "        log_['ID'] = log_['fname'].map(lambda x: f\"{os.path.basename(os.path.dirname(x))}.nii.gz\")\n",
    "    else:\n",
    "        log_['ID'] = log_['fname'].map(lambda x: os.path.basename(os.path.dirname(x)))\n",
    "    return log_\n",
    "\n",
    "def map_mn(x):\n",
    "    return x.replace('densen', 'DenseN').replace('resnet', 'ResNet').replace('vgg', 'VGG').replace('inception_v3', 'InceptionV3')\n",
    "\n",
    "all_log_ = []\n",
    "metrics_dfs = []\n",
    "metric_results = []\n",
    "all_preds = []\n",
    "all_gts = []\n",
    "all_model_names = []\n",
    "\n",
    "model_root = os.path.join(get_param_in_cwd('model_root'), 'semi_models')\n",
    "for model in os.listdir(model_root):\n",
    "    all_pred = []\n",
    "    all_gt = []\n",
    "    all_groups = []\n",
    "    sel_epoch = get_param_in_cwd('sel_epoch')['semi']\n",
    "    if sel_epoch is None:\n",
    "        val_log = pd.concat([get_log(os.path.join(model_root , model, f\"viz/BST_TRAIN_RESULTS.txt\")),\n",
    "                            get_log(os.path.join(model_root, model, f\"viz/BST_VAL_RESULTS.txt\"))], axis=0)\n",
    "    else:\n",
    "        val_log = pd.concat([get_log(os.path.join(model_root, model, f\"train/Epoch-{sel_epoch}.txt\")),\n",
    "                            get_log(os.path.join(model_root, model, f\"valid/Epoch-{sel_epoch}.txt\"))], axis=0)\n",
    "    val_log = pd.merge(val_log, group_info, on='ID', how='inner')\n",
    "    val_log['model'] = f\"{model}\"\n",
    "    ug_groups = get_param_in_cwd('subsets')\n",
    "    ul_labels = np.unique(val_log['pred_label'])\n",
    "    for g in ug_groups:\n",
    "        sub_group = val_log[val_log['group'] == g]\n",
    "        sub_group['label-1'] = list(map(lambda x: x[0] if x[1] == 1 else 1-x[0], \n",
    "                                        np.array(sub_group[['pred_score', 'pred_label']])))\n",
    "        sub_group['label-0'] = 1 - sub_group['label-1']\n",
    "        sub_group = normalize_df(sub_group, not_norm=[c for c in sub_group.columns if c != 'label-1'], method='minmax')\n",
    "        all_groups.append(g)                    \n",
    "        all_log_.append(sub_group)\n",
    "        for ul in [1]:\n",
    "            pred_score = np.array(sub_group['label-1']) \n",
    "            gt = [1 if gt_ == ul else 0 for gt_ in np.array(sub_group['gt'])]\n",
    "            acc, auc, ci, tpr, tnr, ppv, npv, _, _, _, thres = metrics.analysis_pred_binary(gt, pred_score, use_youden=True)\n",
    "            ci = f\"{ci[0]:.4f}-{ci[1]:.4f}\"\n",
    "            metric_results.append([model, acc, auc, ci, tpr, tnr, ppv, npv, thres, g])\n",
    "            all_pred.append(pred_score)\n",
    "            all_gt.append(gt)\n",
    "    # 绘制每个模型的ROC\n",
    "    draw_roc(all_gt, all_pred, labels=all_groups, title=f'Model: {map_mn(model)}')\n",
    "    plt.savefig(f'img/PatchSemi_{model}_roc.svg', bbox_inches='tight')\n",
    "    plt.show()\n",
    "    # 整合到所有模型汇总。\n",
    "    all_preds.extend(all_pred)\n",
    "    all_gts.extend(all_gt)\n",
    "    all_model_names.append(model)\n",
    "for gi, g in enumerate(all_groups):\n",
    "    draw_roc(all_gts[gi::len(all_groups)], all_preds[gi::len(all_groups)], \n",
    "             labels=[map_mn(m) for m in all_model_names], \n",
    "             title=f\"Cohort {g}\")\n",
    "    plt.savefig(f'img/PatchSemi_{g}_roc.svg', bbox_inches='tight')\n",
    "    plt.show()\n",
    "metrics_df = pd.DataFrame(metric_results, \n",
    "                          columns=['ModelName', 'Acc', 'AUC', '95% CI', 'Sensitivity', 'Specificity', 'PPV', 'NPV', \n",
    "                                   'Youden', 'Cohort'])\n",
    "display(metrics_df)\n",
    "metrics_dfs.append(metrics_df)\n",
    "pd.concat(metrics_dfs, axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a45a539",
   "metadata": {},
   "source": [
    "# 保存预测结果\n",
    "\n",
    "将深度学习的预测结果，保存中与组学的预测结果相同的格式，f1便后续进行汇总。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06cc4c38",
   "metadata": {},
   "outputs": [],
   "source": [
    "sel_model = get_param_in_cwd('sel_model', model)\n",
    "\n",
    "all_logs = pd.concat(all_log_, axis=0)\n",
    "sel_log = all_logs[all_logs['model'].str.contains(sel_model['semi'])]\n",
    "sel_log[['ID', 'label-1', 'pred_label', 'gt']].to_csv('results/PathSemi_DL_PREDICTIONS.csv', index=False)\n",
    "sel_log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "933925dc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
